package com.itcast.spark.baseFeaturation

import org.apache.spark.ml.feature._
import org.apache.spark.ml.linalg.Vectors
import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.{SparkConf, SparkContext}

/**
 * DESC:目的是用于连续值属性的离散化
 * Buckrizer操作---分箱
 */
object _03NumbericDataTest {
  def main(args: Array[String]): Unit = {
    //这里是准备环境
    val conf: SparkConf = new SparkConf().setAppName("_04RandomNumber").setMaster("local[*]")
    val spark: SparkSession = SparkSession.builder().config(conf).getOrCreate()
    val sc: SparkContext = spark.sparkContext
    sc.setLogLevel("WARN")
    val splits: Array[Double] = Array(Double.NegativeInfinity, -8, 0, 8, Double.PositiveInfinity)
    //这里是准备数据
    val dataFrame = spark.createDataFrame(Seq(
      (0, Vectors.dense(1.0, 0.5, -1.0)),
      (1, Vectors.dense(2.0, 1.0, 1.0)),
      (2, Vectors.dense(4.0, 10.0, 2.0))
    )).toDF("id", "features")
    //fit----就是训练val indexerModel: StringIndexerModel = indexer.fit(df)----有了Model才能对其他数据预测
    //tranform就是一个预测的过程----使用模型预测
    //但是在这里的特征工程中，如何理解？
    //1-如果一个操作实继承自Estimator的就会实现fit在tranform的方法
    //2-如果一个操作继承自Tanfromer的类就需要直接实现tranfrom方法
    //归一化---(x-min)/(max-min)
    val minMaxScaler: MinMaxScaler = new MinMaxScaler().setInputCol("features").setOutputCol("minmaxfeatures")
    val minMaxScalerModel: MinMaxScalerModel = minMaxScaler.fit(dataFrame)
    minMaxScalerModel.transform(dataFrame).show(false)
    //归一化---(x)/(max(|x|))
    val maxAbsScaler: MaxAbsScaler = new MaxAbsScaler().setInputCol("features").setOutputCol("maxabsfeatures")
    val maxAbsScalerModel: MaxAbsScalerModel = maxAbsScaler.fit(dataFrame)
    maxAbsScalerModel.transform(dataFrame).show()
    //（x-avg）/variance    by removing the mean and scaling to unit variance
    val scaler: StandardScaler = new StandardScaler().setInputCol("features").setOutputCol("standsclerModel")
    scaler.fit(dataFrame).transform(dataFrame).show(false)
  }
}
